# -*- coding: utf-8 -*-

import pprint
import random
import time
from urllib.parse import urlencode

import execjs
import requests
import scrapy
import json
from BiliBiliSpider.items import YoukuSpiderItem
from scrapy.http.cookies import CookieJar
import re
from BiliBiliSpider.spiders.userAgent import USER_AGENT

YOUKU_HEADERS = {
    "Accept": "*/*",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Connection": "keep-alive",
    'Host': 'acs.youku.com',
    'sec-ch-ua': '" Not;A Brand";v="99", "Google Chrome";v="97", "Chromium";v="97"',
    "Sec-Fetch-Dest": "script",
    "Sec-Fetch-Mode": "no-cors",
    "Sec-Fetch-Site": "same-site",
    'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10'
}

YOUKU_HEADERS_USE = {
    'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10'
}

PARAMS = {
    "jsv": "2.5.8",
    "appKey": "24679788",
    "t": '',
    "sign": '',
    "api": "mtop.youku.play.ups.appinfo.get",
    "v": "1.1",
    "timeout": "20000",
    "YKPid": "20160317PLF000211",
    "YKLoginRequest": "true",
    "AntiFlood": "true",
    "AntiCreep": "true",
    "type": "jsonp",
    "dataType": "jsonp",
    "callback": "mtopjsonp1",
    "data": '',
}

# 需要添加时间戳、视频vid、视频的showid  后续参数的拼接--> data % (time, vid, showid)
DATA = r'''{"steal_params":"{\"ccode\":\"0502\",\"client_ip\":\"192.168.1.1\",\"utid\":\"XcC3F4n7ThoCAVQRIgiibcOp\",\"client_ts\":%s,\"version\":\"2.1.69\",\"ckey\":\"DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND\"}","biz_params":"{\"vid\":\"%s\",\"play_ability\":16782592,\"current_showid\":\"%s\",\"preferClarity\":99,\"extag\":\"EXT-X-PRIVINF\",\"master_m3u8\":1,\"media_type\":\"standard,subtitle\",\"app_ver\":\"2.1.69\",\"h265\":1}","ad_params":"{\"vs\":\"1.0\",\"pver\":\"2.1.69\",\"sver\":\"2.0\",\"site\":1,\"aw\":\"w\",\"fu\":0,\"d\":\"0\",\"bt\":\"pc\",\"os\":\"win\",\"osv\":\"10\",\"dq\":\"auto\",\"atm\":\"\",\"partnerid\":\"null\",\"wintype\":\"interior\",\"isvert\":0,\"vip\":1,\"p\":1,\"rst\":\"mp4\",\"needbf\":2,\"avs\":\"1.0\"}"}'''
URL = 'https://acs.youku.com/h5/mtop.youku.play.ups.appinfo.get/1.1/?jsv=2.5.8&appKey=24679788&t={}&sign={}&data={}'
END_URL = 'https://acs.youku.com/h5/mtop.youku.play.ups.appinfo.get/1.1/?'

with open('./youkuJs.js', 'r', encoding='utf-8') as f:
    jsCode = f.read()


class YoukuSpider(scrapy.Spider):
    name = 'youku'

    # 多个爬虫文件多个item管道时，需要指定各个爬虫文件所对应的item管道，否则不生效
    custom_settings = {
        # 'DOWNLOADER_MIDDLEWARES': {
        #     'BiliBiliSpider.middlewares.RandomProxyDownloaderMiddlewareYouku': 300,
        # },
        'ITEM_PIPELINES': {
            'BiliBiliSpider.pipelines.DownloadYkVideoPipeline': 310,
            'BiliBiliSpider.pipelines.YouKuSaveMysqlPipeline': 351,
            'scrapy_redis.pipelines.RedisPipeline': 300,
        },
    }

    def start_requests(self):
        urls = 'https://www.youku.com/'
        YOUKU_HEADERS_USE['User-Agent'] = random.choice(USER_AGENT)
        item = YoukuSpiderItem()
        item['ykReferer'] = urls
        yield scrapy.Request(
            url=urls,
            meta={'item': item},
            callback=self.oneYkHtml,
            headers=YOUKU_HEADERS_USE,
        )

    def oneYkHtml(self, response):
        """解析一级页面，提取优酷视频各个频道的url"""
        oneHtml = response.text
        item = response.meta['item']
        try:
            youkuYzm = re.findall("msg: '(.*?)',", oneHtml, re.S)[0].replace(" ", "")
            if youkuYzm == '优酷web页面-验证码-临时':
                with open('1.html', encoding='utf-8') as f:
                    f.write(oneHtml)
                return
        except:
            print('请求头', response.request.headers)
            print('可能访问被拦截', oneHtml)
            # 提取各个频道url
            indexUrlList = re.findall('<a class="homepage_sub_title_new" href="(.*?)"', oneHtml, re.S)
            indexUrlList = list(set(indexUrlList))  # 给url去重
            print('初始列表', indexUrlList)
            time.sleep(random.randrange(1, 5))
            YOUKU_HEADERS_USE['User-Agent'] = random.choice(USER_AGENT)
            YOUKU_HEADERS_USE['Referer'] = item['ykReferer']
            for index in range(0, 2):
                item['ykReferer'] = indexUrlList[index]
                yield scrapy.Request(
                    url=indexUrlList[index],
                    meta={'item': item},
                    headers=YOUKU_HEADERS_USE,
                    callback=self.twoYkHtml,
                )

    def twoYkHtml(self, response):
        """解析二级页面，提取优酷各个视频播放地址url, 同样需要拼接https:"""
        towHtml = response.text
        item = response.meta['item']
        try:
            youkuYzm = re.findall("msg: '(.*?)',", towHtml, re.S)[0].replace(" ", "")
            if youkuYzm == '优酷web页面-验证码-临时':
                with open('2.html', encoding='utf-8') as f:
                    f.write(towHtml)
                return
        except:
            print('请求头', response.request.headers)
            print('可能访问被拦截', towHtml)
            # 提取各个视频url
            allVideoList = re.findall('"videoId":"(.*?)"', towHtml, re.S)
            allVideoList = list(set(allVideoList))
            for vid in range(0, len(allVideoList)):
                allVideoList[vid] = 'https://v.youku.com/v_show/id_' + allVideoList[vid] + '.html'
            print('视频列表', allVideoList)
            YOUKU_HEADERS_USE['User-Agent'] = random.choice(USER_AGENT)
            YOUKU_HEADERS_USE['Referer'] = item['ykReferer']
            time.sleep(random.randrange(1, 5))
            for index in range(0, 3):
                item['ykReferer'] = allVideoList[index]
                yield scrapy.Request(
                    url=allVideoList[index],
                    meta={'item': item},
                    headers=YOUKU_HEADERS_USE,
                    callback=self.threeYkHtml,
                )

    def threeYkHtml(self, response):
        """
        解析三级页面, 提取选集页面url，判断视频类型是否是电影或音乐
        """
        threeHtml = response.text
        item = response.meta['item']
        try:
            youkuYzm = re.findall("msg: '(.*?)',", threeHtml, re.S)[0].replace(" ", "")
            if youkuYzm == '优酷web页面-验证码-临时':
                with open('3.html', encoding='utf-8') as f:
                    f.write(threeHtml)
                return
        except:
            print('请求头', response.request.headers)
            print('进入', threeHtml)
            # 提取视频的类型，并去除多余的空格
            try:
                videoName = re.findall("catName: '(.*?)',", threeHtml, re.S)[0].replace(" ", "")
                # 如果视频的类型是电影的话，就不需要进入选集页面
                times = str(int(time.time() * 1000))
                if videoName == '电影' or videoName == '音乐':
                    # 这是视频播放的显示id，请求里面的data需要
                    item['ykCurrentShowId'] = re.findall("showid: '(.*?)',", threeHtml, re.S)[0]
                    # 这是视频播放的vid，请求里面的data和url拼接需要
                    item['ykCurrentVid'] = re.findall("currentEncodeVid: '(.*?)'", threeHtml, re.S)[0]
                    # 视频的标题名称
                    item['ykVideoName'] = re.findall('<meta name="irAlbumName" content="(.*?)"', threeHtml, re.S)[
                        0].replace(" ", "")
                    # 视频的集数
                    item['ykNumber'] = re.findall('"subtitleType":"PLAY_VV","stage":(.*?),', threeHtml, re.S)[0]
                    # 还需要拿到Referer  <link rel="canonical" href="" 这是获取Referer的正则
                    YOUKU_HEADERS['Referer'] = re.findall('<link rel="canonical" href="(.*?html)', threeHtml, re.S)[0]
                    # 这里是获取视频详情及下载地址的url
                    time.sleep(random.randrange(1, 5))
                    data = DATA % (times[0:10], item['ykCurrentVid'], item['ykCurrentShowId'])
                    print('看data', data)
                    urls = URL.format(times, '', data)
                    YOUKU_HEADERS['User-Agent'] = random.choice(USER_AGENT)
                    YOUKU_HEADERS['Referer'] = item['ykReferer']
                    # 先请求一次，获取cookie 和 token
                    res = requests.get(url=urls, headers=YOUKU_HEADERS)
                    print('res的请求', res.text)
                    dictCookie = requests.utils.dict_from_cookiejar(res.cookies)
                    cookie = '_m_h5_tk=' + dictCookie['_m_h5_tk'] + '; _m_h5_tk_enc=' + dictCookie['_m_h5_tk_enc']
                    YOUKU_HEADERS['cookie'] = cookie
                    token = re.findall('(.*?)_', dictCookie['_m_h5_tk'], re.S)[0]
                    print("cookie and token", cookie, '\n', 'token', token)
                    sign = execjs.compile(jsCode).call('getSign', token + "&" + times + "&" + '24679788' + "&" + data)
                    time.sleep(random.randrange(1, 5))
                    PARAMS['t'] = times
                    PARAMS['sign'] = sign
                    PARAMS['data'] = data
                    res1 = requests.get(url=END_URL, headers=YOUKU_HEADERS, params=PARAMS)
                    print('最终结果', res1.text)
                    m3u8Link = re.findall('"m3u8_url":"(.*?)"', res1.text, re.S)
                    item['ykVideoM3u8'] = m3u8Link[0]
                    item['ykImageLink'] = re.findall('"show_thumburl_big_jpg":"(.*?)"', res1.text, re.S)[0]
                    print('m3u8视频1', m3u8Link)
                    yield item
                else:
                    # 提取优酷视频的选集页面url，电影和音乐不需要找集数
                    videoName = re.findall('"pvv_vid":"(.*?)"', threeHtml, re.S)
                    videoName = list(set(videoName))
                    print('视频VID', videoName)
                    YOUKU_HEADERS_USE['Referer'] = item['ykReferer']
                    for vid in videoName:
                        url = 'https://v.youku.com/v_show/id_' + vid + '.html'
                        print('url--->', url)
                        time.sleep(random.randrange(1, 5))
                        item['ykReferer'] = url
                        yield scrapy.Request(
                            url=url,
                            headers=YOUKU_HEADERS_USE,
                            meta={'item': item},
                            callback=self.selectNum,
                        )
            except:
                pass

    def selectNum(self, response):
        """
        不是电影和音乐就代表有集数，在这里进行集数的遍历请求
        """
        numInfo = response.text
        item = response.meta['item']
        try:
            youkuYzm = re.findall("msg: '(.*?)',", numInfo, re.S)[0].replace(" ", "")
            if youkuYzm == '优酷web页面-验证码-临时':
                with open('4.html', encoding='utf-8') as f:
                    f.write(numInfo)
                return
        except:
            times = str(time.time() * 1000)
            item['ykCurrentShowId'] = re.findall("showid: '(.*?)',", numInfo, re.S)[0]
            # 这是视频播放的vid，请求里面的data和url拼接需要
            item['ykCurrentVid'] = re.findall("currentEncodeVid: '(.*?)'", numInfo, re.S)[0]
            # 视频的标题名称
            item['ykVideoName'] = re.findall('<meta name="irAlbumName" content="(.*?)"', numInfo, re.S)[0].replace(" ",
                                                                                                                   "")
            # 视频集数
            item['ykNumber'] = re.findall('"subtitleType":"PLAY_VV","stage":(.*?),', numInfo, re.S)[0]
            # 这里是获取视频详情及下载地址的url
            time.sleep(random.randrange(1, 5))
            data = DATA % (times[0:10], item['ykCurrentVid'], item['ykCurrentShowId'])
            urls = URL.format(times, '', data)
            YOUKU_HEADERS['User-Agent'] = random.choice(USER_AGENT)
            YOUKU_HEADERS['Referer'] = re.findall('<link rel="canonical" href="(.*?html)', numInfo, re.S)[0]
            # 先请求一次，获取cookie 和 token
            res = requests.get(url=urls, headers=YOUKU_HEADERS)
            print('res的请求', res.text)
            print('看data1', data)
            dictCookie = requests.utils.dict_from_cookiejar(res.cookies)
            cookie = '_m_h5_tk=' + dictCookie['_m_h5_tk'] + '; _m_h5_tk_enc=' + dictCookie['_m_h5_tk_enc']
            YOUKU_HEADERS['cookie'] = cookie
            token = re.findall('(.*?)_', dictCookie['_m_h5_tk'], re.S)[0]
            print("cookie and token 1", cookie, '\n', 'token', token)
            sign = execjs.compile(jsCode).call('getSign', token + "&" + times + "&" + '24679788' + "&" + data)
            time.sleep(random.randrange(1, 5))
            PARAMS['t'] = times
            PARAMS['sign'] = sign
            PARAMS['data'] = data
            res1 = requests.get(url=END_URL, headers=YOUKU_HEADERS, params=PARAMS)
            print('最终结果', res1.text)
            m3u8Link = re.findall('"m3u8_url":"(.*?)"', res1.text, re.S)
            print('m3u8视频', m3u8Link)
            if not m3u8Link:
                return
            item['ykVideoM3u8'] = m3u8Link[0]
            item['ykImageLink'] = re.findall('"show_thumburl_big_jpg":"(.*?)"', res1.text, re.S)[0]
            yield item
